

<!DOCTYPE html>
<html lang="zh-CN" data-default-color-scheme=auto>



<head>
  <meta charset="UTF-8">
  <link rel="apple-touch-icon" sizes="76x76" href="/img/fluid.png">
  <link rel="icon" href="/img/fluid.png">
  <meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=5.0, shrink-to-fit=no">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  
  <meta name="theme-color" content="#2f4154">
  <meta name="author" content="John Doe">
  <meta name="keywords" content="">
  
    <meta name="description" content="@[toc] 参考资料  视频链接 Linux scp命令  一、前言 在Hadoop完全分布式集群环境下，里面的各个节点都是通过 SSH免密登陆 连接的，比如现有三台集群节点，分别是 hadoop101、hadoop102、hadoo103。 hadoop101节点可以通过 ssh命令连接到hadoop102，即： 1ssh hadoop102  那么，除了远程登陆以外，还有其他的作用吗？ 当然">
<meta property="og:type" content="article">
<meta property="og:title" content="Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令">
<meta property="og:url" content="http://example.com/2022/01/15/Hadoop%20_%20%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE%EF%BC%88%E4%B8%80%EF%BC%89%E4%BD%BF%E7%94%A8Shell%E8%84%9A%E6%9C%AC%E8%A7%A3%E5%86%B3%E5%AE%8C%E5%85%A8%E5%88%86%E5%B8%83%E5%BC%8F%E9%9B%86%E7%BE%A4%E8%8A%82%E7%82%B9%E4%B9%8B%E9%97%B4%E7%9A%84%E6%96%87%E4%BB%B6%E4%BC%A0%E8%BE%93%E9%97%AE%E9%A2%98%20_%20scp%E5%91%BD%E4%BB%A4%20_%20rsync%E5%91%BD%E4%BB%A4/index.html">
<meta property="og:site_name" content="Hexo">
<meta property="og:description" content="@[toc] 参考资料  视频链接 Linux scp命令  一、前言 在Hadoop完全分布式集群环境下，里面的各个节点都是通过 SSH免密登陆 连接的，比如现有三台集群节点，分别是 hadoop101、hadoop102、hadoo103。 hadoop101节点可以通过 ssh命令连接到hadoop102，即： 1ssh hadoop102  那么，除了远程登陆以外，还有其他的作用吗？ 当然">
<meta property="og:locale" content="zh_CN">
<meta property="article:published_time" content="2022-01-15T12:40:47.000Z">
<meta property="article:modified_time" content="2022-08-22T15:43:53.432Z">
<meta property="article:author" content="John Doe">
<meta property="article:tag" content="Hadoop">
<meta name="twitter:card" content="summary_large_image">
  
  
  
  <title>Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令 - Hexo</title>

  <link  rel="stylesheet" href="https://lib.baomitu.com/twitter-bootstrap/4.6.1/css/bootstrap.min.css" />



  <link  rel="stylesheet" href="https://lib.baomitu.com/github-markdown-css/4.0.0/github-markdown.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/hint.css/2.7.0/hint.min.css" />

  <link  rel="stylesheet" href="https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.css" />



<!-- 主题依赖的图标库，不要自行修改 -->
<!-- Do not modify the link that theme dependent icons -->

<link rel="stylesheet" href="//at.alicdn.com/t/font_1749284_hj8rtnfg7um.css">



<link rel="stylesheet" href="//at.alicdn.com/t/font_1736178_lbnruvf0jn.css">


<link  rel="stylesheet" href="/css/main.css" />


  <link id="highlight-css" rel="stylesheet" href="/css/highlight.css" />
  
    <link id="highlight-css-dark" rel="stylesheet" href="/css/highlight-dark.css" />
  




  <script id="fluid-configs">
    var Fluid = window.Fluid || {};
    Fluid.ctx = Object.assign({}, Fluid.ctx)
    var CONFIG = {"hostname":"example.com","root":"/","version":"1.9.2","typing":{"enable":true,"typeSpeed":70,"cursorChar":"_","loop":false,"scope":[]},"anchorjs":{"enable":true,"element":"h1,h2,h3,h4,h5,h6","placement":"left","visible":"hover","icon":""},"progressbar":{"enable":true,"height_px":3,"color":"#29d","options":{"showSpinner":false,"trickleSpeed":100}},"code_language":{"enable":true,"default":"TEXT"},"copy_btn":true,"image_caption":{"enable":true},"image_zoom":{"enable":true,"img_url_replace":["",""]},"toc":{"enable":true,"placement":"right","headingSelector":"h1,h2,h3,h4,h5,h6","collapseDepth":0},"lazyload":{"enable":true,"loading_img":"/img/loading.gif","onlypost":false,"offset_factor":2},"web_analytics":{"enable":false,"follow_dnt":true,"baidu":null,"google":null,"gtag":null,"tencent":{"sid":null,"cid":null},"woyaola":null,"cnzz":null,"leancloud":{"app_id":null,"app_key":null,"server_url":null,"path":"window.location.pathname","ignore_local":false}},"search_path":"/local-search.xml"};

    if (CONFIG.web_analytics.follow_dnt) {
      var dntVal = navigator.doNotTrack || window.doNotTrack || navigator.msDoNotTrack;
      Fluid.ctx.dnt = dntVal && (dntVal.startsWith('1') || dntVal.startsWith('yes') || dntVal.startsWith('on'));
    }
  </script>
  <script  src="/js/utils.js" ></script>
  <script  src="/js/color-schema.js" ></script>
  


  
<meta name="generator" content="Hexo 6.2.0"></head>


<body>
  

  <header>
    

<div class="header-inner" style="height: 70vh;">
  <nav id="navbar" class="navbar fixed-top  navbar-expand-lg navbar-dark scrolling-navbar">
  <div class="container">
    <a class="navbar-brand" href="/">
      <strong>尤 Ni&#39;s Blog</strong>
    </a>

    <button id="navbar-toggler-btn" class="navbar-toggler" type="button" data-toggle="collapse"
            data-target="#navbarSupportedContent"
            aria-controls="navbarSupportedContent" aria-expanded="false" aria-label="Toggle navigation">
      <div class="animated-icon"><span></span><span></span><span></span></div>
    </button>

    <!-- Collapsible content -->
    <div class="collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav ml-auto text-center">
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/">
                <i class="iconfont icon-home-fill"></i>
                首页
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/archives/">
                <i class="iconfont icon-archive-fill"></i>
                归档
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/categories/">
                <i class="iconfont icon-category-fill"></i>
                分类
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/tags/">
                <i class="iconfont icon-tags-fill"></i>
                标签
              </a>
            </li>
          
        
          
          
          
          
            <li class="nav-item">
              <a class="nav-link" href="/about/">
                <i class="iconfont icon-user-fill"></i>
                关于
              </a>
            </li>
          
        
        
          <li class="nav-item" id="search-btn">
            <a class="nav-link" target="_self" href="javascript:;" data-toggle="modal" data-target="#modalSearch" aria-label="Search">
              &nbsp;<i class="iconfont icon-search"></i>&nbsp;
            </a>
          </li>
          
        
        
          <li class="nav-item" id="color-toggle-btn">
            <a class="nav-link" target="_self" href="javascript:;" aria-label="Color Toggle">&nbsp;<i
                class="iconfont icon-dark" id="color-toggle-icon"></i>&nbsp;</a>
          </li>
        
      </ul>
    </div>
  </div>
</nav>

  

<div id="banner" class="banner" parallax=true
     style="background: url('/img/default.png') no-repeat center center; background-size: cover;">
  <div class="full-bg-img">
    <div class="mask flex-center" style="background-color: rgba(0, 0, 0, 0.3)">
      <div class="banner-text text-center fade-in-up">
        <div class="h2">
          
            <span id="subtitle" data-typed-text="Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令"></span>
          
        </div>

        
          
  <div class="mt-3">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-author" aria-hidden="true"></i>
        John Doe
      </span>
    
    
      <span class="post-meta">
        <i class="iconfont icon-date-fill" aria-hidden="true"></i>
        <time datetime="2022-01-15 20:40" pubdate>
          2022年1月15日 晚上
        </time>
      </span>
    
  </div>

  <div class="mt-1">
    
      <span class="post-meta mr-2">
        <i class="iconfont icon-chart"></i>
        
          4k 字
        
      </span>
    

    
      <span class="post-meta mr-2">
        <i class="iconfont icon-clock-fill"></i>
        
        
        
          34 分钟
        
      </span>
    

    
    
  </div>


        
      </div>

      
    </div>
  </div>
</div>

</div>

  </header>

  <main>
    
      

<div class="container-fluid nopadding-x">
  <div class="row nomargin-x">
    <div class="side-col d-none d-lg-block col-lg-2">
      

    </div>

    <div class="col-lg-8 nopadding-x-md">
      <div class="container nopadding-x-md" id="board-ctn">
        <div id="board">
          <article class="post-content mx-auto">
            <!-- SEO header -->
            <h1 style="display: none">Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令</h1>
            
              <p class="note note-info">
                
                  
                    本文最后更新于：1 小时前
                  
                
              </p>
            
            
              <div class="markdown-body">
                
                <p>@[toc]</p>
<h1 id="参考资料"><a href="#参考资料" class="headerlink" title="参考资料"></a>参考资料</h1><hr>
<ul>
<li><a target="_blank" rel="noopener" href="https://www.bilibili.com/video/BV1Qp4y1n7EN?p=28&spm_id_from=pageDriver">视频链接</a></li>
<li><a target="_blank" rel="noopener" href="https://www.runoob.com/linux/linux-comm-scp.html">Linux scp命令</a></li>
</ul>
<h1 id="一、前言"><a href="#一、前言" class="headerlink" title="一、前言"></a>一、前言</h1><hr>
<p>在Hadoop完全分布式集群环境下，里面的各个节点都是通过 SSH免密登陆 连接的，比如现有三台集群节点，分别是 hadoop101、hadoop102、hadoo103。</p>
<p>hadoop101节点可以通过 ssh命令连接到hadoop102，即：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">ssh hadoop102<br></code></pre></td></tr></table></figure>

<p>那么，除了远程登陆以外，还有其他的作用吗？</p>
<p>当然有的，比如远程传输文件：<code>scp命令</code><a target="_blank" rel="noopener" href="https://www.runoob.com/linux/linux-comm-scp.html">参考资料</a></p>
<p>如果要将hadoop101节点的hadoop配置文件传输到hadoop102，那么可以尝试：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">scp -r /opt/module/hadoop/etc/hadoop/*.xml root@hadoop102:/opt/module/hadoop/etc/hadoop<br></code></pre></td></tr></table></figure>

<p>但是如果每次都这样使用scp命令去传输，会比较浪费时间，因为这两个集群节点的hadoop目录都是相同的，只是配置文件的内容不同，接下来将根据参考的资料进行描述，如何使用一些命令来简化集群之间的文件传输。</p>
<h1 id="二、rsync-远程同步工具"><a href="#二、rsync-远程同步工具" class="headerlink" title="二、rsync 远程同步工具"></a>二、rsync 远程同步工具</h1><hr>
<p>rsync 主要用于 <code>备份</code> 和 <code>镜像</code>。它具有速度快、避免复制相同内容和支持符号链接的优点。<br>rsync 和 scp的区别： 用 rsync 做文件复制要比scp的速度快，rsync只对差异文件做更新，scp是复制所有的文件。</p>
<p><em><strong>基本语法</strong></em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs shell">rsync -av $pdir/$fname $user@host:$pdir/$fname<br><span class="hljs-meta prompt_"># </span><span class="language-bash">命令 选项参数 要拷贝的文件路径/名称 目的地用户@主机名:目的地路径/名称</span><br></code></pre></td></tr></table></figure>
<p><em><strong>选项参数的说明：</strong></em><br><code>-a</code> ——归档拷贝   <code>-v</code> —— 显示复制过程</p>
<h1 id="三、使用-shell-编写分发脚本"><a href="#三、使用-shell-编写分发脚本" class="headerlink" title="三、使用 shell 编写分发脚本"></a>三、使用 shell 编写分发脚本</h1><hr>
<p>需求：循环复制文件到所有节点的相同目录下<br>需求分析：</p>
<ul>
<li>由于rsync命令速度比scp命令快，所以脚本使用rsync命令实现</li>
<li>在文件复制时候，后者的路径少一级，比如在前言部分中，复制hadoop的配置文件，前者需指定<code>/*.xml</code>参数，后者则不需要指定这个<code>/*.xml</code></li>
<li>脚本在任何路径都能使用，故shell脚本需方在声明了全局环境变量的路径比如：<code>~/bin</code></li>
</ul>
<p><em><strong>具体实现：</strong></em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs shell">cd ~/bin<br>sudo vim xsync<br></code></pre></td></tr></table></figure>
<p><em><strong>脚本内容：（参考视频资料）</strong></em></p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><code class="hljs shell"><span class="hljs-meta prompt_">#</span><span class="language-bash">!/bin/bash</span><br><span class="hljs-meta prompt_"></span><br><span class="hljs-meta prompt_">#</span><span class="language-bash">1. 判断参数个数</span><br>if [ $# -lt 1 ]<br>then<br>    echo 请输入合理的一个参数:[要拷贝的文件/目录]<br>    exit;<br>fi<br><span class="hljs-meta prompt_">#</span><span class="language-bash">2. 遍历集群所有机器</span><br>for host in hadoop101 hadoop102 hadoop103<br>do<br>    echo &#x27;====================  集群节点$host的操作  ====================&#x27;<br>    #3. 遍历所有目录，逐个<br>    for file in $@<br>    do<br>        #4. 判断文件是否存在<br>        if [ -e $file ]<br>            then<br>                #5. 获取父目录<br>                pdir=$(cd -P $(dirname $file); pwd)<br>                #6. 获取当前文件的名称<br>                fname=$(basename $file)<br>                ssh $host &quot;mkdir -p $pdir&quot;<br>                rsync -av $pdir/$fname $host:$pdir<br>            else<br>                echo &#x27;$file 文件不存在!&#x27;<br>        fi<br>    done<br>done<br></code></pre></td></tr></table></figure>
<p>创建脚本后需要对其赋予可执行权限:</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">sudo chmod +x xsync<br></code></pre></td></tr></table></figure>
<p>接下来笔者结合视频讲解还有自己的理解对这个脚本进行解读</p>
<p>在分发文件时有两种情况，第一种分发一个具体文件，第二种分发整个文件夹，由此产生了几个问题：</p>
<ul>
<li>脚本是同步到哪几个节点？</li>
<li>如果接收文件节点的目录不存在怎么办？</li>
<li>如果一次性分发多个文件或者目录怎么办？</li>
</ul>
<p>首先脚本的定位就是分发到所有的集群节点，这一点可以确认，所以就需for循环遍历所有的节点，</p>
<p>其次，为防止接收文件的位置目录不存在，需在接收文件的节点里创建这个文件的目录，同时获取文件的目录，并且要获取文件的具体名称，将这二者区分开则可以解决接收文件节点不存在的问题。</p>
<p>一次性分发多个文件，使用shell脚本的#@ 以个体的形式获取所有的输入参数，然后使用 for in 进行循环遍历即可。</p>
<p>如果对 shell 脚本不够熟悉，可以参考这两篇文章，里面有写参考的资料，查看参考资料会更加清晰明了：</p>
<ul>
<li><p><a target="_blank" rel="noopener" href="https://blog.csdn.net/Unirithe/article/details/122481068">Linux | Shell 学习笔记（一）Shell概述 | Linux系统自带的解析器&#x2F;bin&#x2F;bash | bash&#x2F;sh执行脚本 | 脚本中$n 、$#等特殊变量 | condition条件判断</a></p>
</li>
<li><p><a target="_blank" rel="noopener" href="https://blog.csdn.net/Unirithe/article/details/122510371">Linux | Shell 学习笔记（二）Shell 流程控制 if、case、for、while| read读取输入 | 函数的使用 | cut、sed、awk、sort命令 +Demo</a></p>
</li>
</ul>
<p>笔者认为上面的分发脚本，比较难理解的就是这一部分：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre></td><td class="code"><pre><code class="hljs shell">pdir=$(cd -P $(dirname $file); pwd)<br>fname=$(basename $file)<br>ssh $host &quot;mkdir -p $pdir&quot;<br>rsync -av $pdir/$fname $host:$pdir<br></code></pre></td></tr></table></figure>

<p>笔者最疑惑的就是，<code>cd -P </code> 的意义是什么？ 于是在Linux系统里输入了 <code>help cd</code> 查看cd命令的帮助文档，返回的结果如下：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br></pre></td><td class="code"><pre><code class="hljs shell">[s0125@hadoop ~]$ help cd<br>cd: cd [-L|[-P [-e]]] [dir]<br>    Change the shell working directory.<br>    <br>    Change the current directory to DIR.  The default DIR is the value of the<br>    HOME shell variable.<br>    <br>    The variable CDPATH defines the search path for the directory containing<br>    DIR.  Alternative directory names in CDPATH are separated by a colon (:).<br>    A null directory name is the same as the current directory.  If DIR begins<br>    with a slash (/), then CDPATH is not used.<br>    <br>    If the directory is not found, and the shell option `cdable_vars&#x27; is set,<br>    the word is assumed to be  a variable name.  If that variable has a value,<br>    its value is used for DIR.<br>    <br>    Options:<br>        -L	force symbolic links to be followed<br>        -P	use the physical directory structure without following symbolic<br>    	links<br>        -e	if the -P option is supplied, and the current working directory<br>    	cannot be determined successfully, exit with a non-zero status<br>    <br>    The default is to follow symbolic links, as if `-L&#x27; were specified.<br>    <br>    Exit Status:<br>    Returns 0 if the directory is changed, and if $PWD is set successfully when<br>    -P is used; non-zero otherwise.<br><br></code></pre></td></tr></table></figure>

<p>其中比较关键的信息：</p>
<p><code> 原先：</code><br> <figure class="highlight livecodeserver"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs livecodeserver">-P	use <span class="hljs-keyword">the</span> physical <span class="hljs-built_in">directory</span> structure <span class="hljs-keyword">without</span> following symbolic<br>   	links<br></code></pre></td></tr></table></figure><br> <code>机翻：</code><br> <figure class="highlight css"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs css">-<span class="hljs-selector-tag">P</span> 使用物理目录结构而不使用以下符号链接<br></code></pre></td></tr></table></figure></p>
<p>总之就是理解为找到真实的地址吧</p>
<p>除此之外还用到了 <code>dirname</code>命令，脚本的内容：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">pdir=$(cd -P $(dirname $file); pwd)<br></code></pre></td></tr></table></figure>
<p>dirname命令的作用就是取文件相对于当前的位置目录，比如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">dirname /opt/module/love.txt<br></code></pre></td></tr></table></figure>
<p>执行结果就是 <code>/opt/module</code>，它并不能判断文件是否存在，只是相当于一种字符串处理的操作，提取文件所在目录仅此而已。</p>
<p>而 <code>pwd</code> 就是取当前的目录，脚本的这句就是先切换目录到传输文件的所在目录（dirname获取到的），然后再将当前的文件目录复制给pdir变量，用于表示要拷贝的文件的目录。</p>
<p>接下来的第二行</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">fname=$(basename $file)<br></code></pre></td></tr></table></figure>
<p>basename同样是一种字符串处理的操作，作用是将路径的文件名提取出来，比如：</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><code class="hljs shell">basename /opt/module/love.txt<br></code></pre></td></tr></table></figure>
<p>返回的结果是字符串形式的：<code>love.txt</code></p>
<p>通过以上两行就获取到了拷贝文件的目录位置和文件的名称，接下来就是先创建好目录，防止目录不存在的情况，<code>mkdir -p </code>可以在文件存在时忽略报错，然后<code>ssh</code>命令则可以将后面的命令发送到远程的节点。</p>
<figure class="highlight shell"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><code class="hljs shell">ssh $host &quot;mkdir -p $pdir&quot;<br>rsync -av $pdir/$fname $host:$pdir<br></code></pre></td></tr></table></figure>


<h1 id="四、总结"><a href="#四、总结" class="headerlink" title="四、总结"></a>四、总结</h1><hr>
<p>hadoop集群节点的脚本分发主要是通过ssh免密码登陆、rsync工具，rsync只拷贝有差异的文件，这样可以提高效率，另外还需要熟悉一下shell脚本的for循环遍历，以及条件判断等知识点，大概思路就是先循环遍历所有的集群节点，然后在循环里通过<code>#@</code>遍历每一个输入的参数，即需同步&#x2F;拷贝的文件，对于每个参数，先提取目录，再提取文件名，提取完毕后先使用<code>ssh $host &quot;mkdir -p...&quot;</code> 远程将节点的目录创建好，以免报错，最后再使用<code>rsync</code>命令去拷贝文件。</p>

                
              </div>
            
            <hr/>
            <div>
              <div class="post-metas my-3">
  
    <div class="post-meta mr-3 d-flex align-items-center">
      <i class="iconfont icon-category"></i>
      

<span class="category-chains">
  
  
    
      <span class="category-chain">
        
  <a href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/" class="category-chain-item">大数据</a>
  
  

      </span>
    
  
</span>

    </div>
  
  
    <div class="post-meta">
      <i class="iconfont icon-tags"></i>
      
        <a href="/tags/Hadoop/">#Hadoop</a>
      
    </div>
  
</div>


              
  

  <div class="license-box my-3">
    <div class="license-title">
      <div>Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令</div>
      <div>http://example.com/2022/01/15/Hadoop _ 集群配置（一）使用Shell脚本解决完全分布式集群节点之间的文件传输问题 _ scp命令 _ rsync命令/</div>
    </div>
    <div class="license-meta">
      
        <div class="license-meta-item">
          <div>作者</div>
          <div>John Doe</div>
        </div>
      
      
        <div class="license-meta-item license-meta-date">
          <div>发布于</div>
          <div>2022年1月15日</div>
        </div>
      
      
      <div class="license-meta-item">
        <div>许可协议</div>
        <div>
          
            
            
              <a target="_blank" href="https://creativecommons.org/licenses/by/4.0/">
              <span class="hint--top hint--rounded" aria-label="BY - 署名">
                <i class="iconfont icon-by"></i>
              </span>
              </a>
            
          
        </div>
      </div>
    </div>
    <div class="license-icon iconfont"></div>
  </div>



              
                <div class="post-prevnext my-3">
                  <article class="post-prev col-6">
                    
                    
                      <a href="/2022/01/15/Hadoop%20_%20%E9%9B%86%E7%BE%A4%E9%85%8D%E7%BD%AE%EF%BC%88%E4%BA%8C%EF%BC%89SSH%E5%85%8D%E5%AF%86%E7%99%BB%E9%99%86%E5%8E%9F%E7%90%86%E4%BB%A5%E5%8F%8A%E6%93%8D%E4%BD%9C%20_%20%E5%85%AC%E9%92%A5%E4%B8%8E%E7%A7%81%E9%92%A5/" title="Hadoop _ 集群配置（二）SSH免密登陆原理以及操作 _ 公钥与私钥">
                        <i class="iconfont icon-arrowleft"></i>
                        <span class="hidden-mobile">Hadoop _ 集群配置（二）SSH免密登陆原理以及操作 _ 公钥与私钥</span>
                        <span class="visible-mobile">上一篇</span>
                      </a>
                    
                  </article>
                  <article class="post-next col-6">
                    
                    
                      <a href="/2022/01/15/Linux%20_%20Shell%E7%AC%94%E8%AE%B02%20Shell%20%E6%B5%81%E7%A8%8B%E6%8E%A7%E5%88%B6%20if%E3%80%81case%E3%80%81for%E3%80%81while_%20read%E8%AF%BB%E5%8F%96%E8%BE%93%E5%85%A5%20_%20%E5%87%BD%E6%95%B0%E7%9A%84%E4%BD%BF%E7%94%A8%20_%20cut%E3%80%81sed%E3%80%81awk%E3%80%81sort%E5%91%BD%E4%BB%A4%20+Demo/" title="Linux _ 笔记2 Shell 流程控制 if、case、for、while_ read读取输入 _ 函数的使用 _ cut、sed、awk、sort命令">
                        <span class="hidden-mobile">Linux _ 笔记2 Shell 流程控制 if、case、for、while_ read读取输入 _ 函数的使用 _ cut、sed、awk、sort命令</span>
                        <span class="visible-mobile">下一篇</span>
                        <i class="iconfont icon-arrowright"></i>
                      </a>
                    
                  </article>
                </div>
              
            </div>

            
          </article>
        </div>
      </div>
    </div>

    <div class="side-col d-none d-lg-block col-lg-2">
      
  <aside class="sidebar" style="margin-left: -1rem">
    <div id="toc">
  <p class="toc-header"><i class="iconfont icon-list"></i>&nbsp;目录</p>
  <div class="toc-body" id="toc-body"></div>
</div>



  </aside>


    </div>
  </div>
</div>





  



  



  



  



  







    

    
      <a id="scroll-top-button" aria-label="TOP" href="#" role="button">
        <i class="iconfont icon-arrowup" aria-hidden="true"></i>
      </a>
    

    
      <div class="modal fade" id="modalSearch" tabindex="-1" role="dialog" aria-labelledby="ModalLabel"
     aria-hidden="true">
  <div class="modal-dialog modal-dialog-scrollable modal-lg" role="document">
    <div class="modal-content">
      <div class="modal-header text-center">
        <h4 class="modal-title w-100 font-weight-bold">搜索</h4>
        <button type="button" id="local-search-close" class="close" data-dismiss="modal" aria-label="Close">
          <span aria-hidden="true">&times;</span>
        </button>
      </div>
      <div class="modal-body mx-3">
        <div class="md-form mb-5">
          <input type="text" id="local-search-input" class="form-control validate">
          <label data-error="x" data-success="v" for="local-search-input">关键词</label>
        </div>
        <div class="list-group" id="local-search-result"></div>
      </div>
    </div>
  </div>
</div>

    

    
  </main>

  <footer>
    <div class="footer-inner">
  
    <div class="footer-content">
       <a href="https://hexo.io" target="_blank" rel="nofollow noopener"><span>Hexo</span></a> <i class="iconfont icon-love"></i> <a href="https://github.com/fluid-dev/hexo-theme-fluid" target="_blank" rel="nofollow noopener"><span>Fluid</span></a> 
    </div>
  
  
  
  
</div>

  </footer>

  <!-- Scripts -->
  
  <script  src="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.js" ></script>
  <link  rel="stylesheet" href="https://lib.baomitu.com/nprogress/0.2.0/nprogress.min.css" />

  <script>
    NProgress.configure({"showSpinner":false,"trickleSpeed":100})
    NProgress.start()
    window.addEventListener('load', function() {
      NProgress.done();
    })
  </script>


<script  src="https://lib.baomitu.com/jquery/3.6.0/jquery.min.js" ></script>
<script  src="https://lib.baomitu.com/twitter-bootstrap/4.6.1/js/bootstrap.min.js" ></script>
<script  src="/js/events.js" ></script>
<script  src="/js/plugins.js" ></script>


  <script  src="https://lib.baomitu.com/typed.js/2.0.12/typed.min.js" ></script>
  <script>
    (function (window, document) {
      var typing = Fluid.plugins.typing;
      var subtitle = document.getElementById('subtitle');
      if (!subtitle || !typing) {
        return;
      }
      var text = subtitle.getAttribute('data-typed-text');
      
        typing(text);
      
    })(window, document);
  </script>




  
    <script  src="/js/img-lazyload.js" ></script>
  




  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/tocbot/4.18.2/tocbot.min.js', function() {
    var toc = jQuery('#toc');
    if (toc.length === 0 || !window.tocbot) { return; }
    var boardCtn = jQuery('#board-ctn');
    var boardTop = boardCtn.offset().top;

    window.tocbot.init({
      tocSelector     : '#toc-body',
      contentSelector : '.markdown-body',
      headingSelector : CONFIG.toc.headingSelector || 'h1,h2,h3,h4,h5,h6',
      linkClass       : 'tocbot-link',
      activeLinkClass : 'tocbot-active-link',
      listClass       : 'tocbot-list',
      isCollapsedClass: 'tocbot-is-collapsed',
      collapsibleClass: 'tocbot-is-collapsible',
      collapseDepth   : CONFIG.toc.collapseDepth || 0,
      scrollSmooth    : true,
      headingsOffset  : -boardTop
    });
    if (toc.find('.toc-list-item').length > 0) {
      toc.css('visibility', 'visible');
    }
  });
</script>


  <script src=https://lib.baomitu.com/clipboard.js/2.0.10/clipboard.min.js></script>

  <script>Fluid.plugins.codeWidget();</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/anchor-js/4.3.1/anchor.min.js', function() {
    window.anchors.options = {
      placement: CONFIG.anchorjs.placement,
      visible  : CONFIG.anchorjs.visible
    };
    if (CONFIG.anchorjs.icon) {
      window.anchors.options.icon = CONFIG.anchorjs.icon;
    }
    var el = (CONFIG.anchorjs.element || 'h1,h2,h3,h4,h5,h6').split(',');
    var res = [];
    for (var item of el) {
      res.push('.markdown-body > ' + item.trim());
    }
    if (CONFIG.anchorjs.placement === 'left') {
      window.anchors.options.class = 'anchorjs-link-left';
    }
    window.anchors.add(res.join(', '));
  });
</script>


  
<script>
  Fluid.utils.createScript('https://lib.baomitu.com/fancybox/3.5.7/jquery.fancybox.min.js', function() {
    Fluid.plugins.fancyBox();
  });
</script>


  <script>Fluid.plugins.imageCaption();</script>

  <script  src="/js/local-search.js" ></script>





<!-- 主题的启动项，将它保持在最底部 -->
<!-- the boot of the theme, keep it at the bottom -->
<script  src="/js/boot.js" ></script>


  

  <noscript>
    <div class="noscript-warning">博客在允许 JavaScript 运行的环境下浏览效果更佳</div>
  </noscript>
</body>
</html>
